package com.chance.cc.crawler.development.bootstrap.cctv.search;

import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.filter.FilterUtils;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.development.controller.DevCrawlerController;

public class CCTVSearchStart {

    public static final String domain = "cctv";

    public static final String site = "search";

    public static void main(String[] args) {

        CrawlerRequestRecord requestRecord = CrawlerRequestRecord.builder()
                .startPageRequest(domain, CrawlerEnum.CrawlerRequestType.turnPage) //翻页
                .domain(domain)
                .httpUrl("https://search.cctv.com/search.php?qtext=nike&type=web") //要采集的地址
                .recordKey("https://search.cctv.com/search.php?qtext=nike&type=web") //表示的是唯一，放第二次就不行了
                .releaseTime(System.currentTimeMillis())
                .filter(CrawlerEnum.CrawlerRecordFilter.keyOrDateRange) //过滤条件
                .addFilterInfo(FilterUtils.memoryFilterKeyInfo(domain))  //具体的条件
                .addFilterInfo(FilterUtils.dateRangeFilterInfo(7 * 24, null))
                .build();

        requestRecord.tagsCreator().bizTags().addDomain(domain);
        requestRecord.tagsCreator().bizTags().addSite(site);
        requestRecord.getHttpRequest().addExtra("url", "https://search.cctv.com/search.php?qtext=%s&sort=date&type=web&vtime=&datepid=1&channel=&page=1");

        CrawlerRequestRecord keywordRecord = CrawlerRequestRecord.builder()
                .startPageRequest(domain, CrawlerEnum.CrawlerRequestType.turnPageItem)
                .httpUrl("http://192.168.1.217:9599/v1/meta/cctv/keys?site=search")
                .requestLabelTag(CrawlerEnum.CrawlerRequestType.supportSource)
                .requestLabelTag(CrawlerEnum.CrawlerRequestType.internalDownload)
                .build();

        DevCrawlerController.builder()
                .triggerInfo(domain, domain, System.currentTimeMillis(), domain)
                .crawlerRequestQueue(DevCrawlerController.devRequestQueue(domain))  //管道
                .consoleResultPipeline() //控制台输出
                .fileResultPipeline("D:\\chance\\log\\cctv.log", false) //文件输出
                .requestRecord(requestRecord)
                .supportRecord(keywordRecord)
                .build("com.chance.cc.crawler.development.scripts.cctv.search")
                .start();


    }
}
